#!/bin/bash
#############################################################################
#  Copyright (C) 2013-2015 Lawrence Livermore National Security, LLC.
#  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
#  Written by Albert Chu <chu11@llnl.gov>
#  LLNL-CODE-644248
#
#  This file is part of Magpie, scripts for running Hadoop on
#  traditional HPC systems.  For details, see https://github.com/llnl/magpie.
#
#  Magpie is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  Magpie is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with Magpie.  If not, see <http://www.gnu.org/licenses/>.
#############################################################################

# This script is the core sparkwordcount running script.  For the most
# part, it shouldn't be editted.  See job submission files for
# configuration details.

source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-paths

# This is a job, no loading export files or libs except for minimal convenience ones

sparkwordcountclass="org.apache.spark.examples.JavaWordCount"

cd ${SPARK_HOME}

# For some reason, the examples jar is named with seemingly random
# versioning, so we'll have to "find" it for each case

if [ "${SPARK_SPARKWORDCOUNT_COPY_IN_FILE}X" != "X" ]
then
    if echo ${SPARK_SPARKWORDCOUNT_FILE} | grep -q -E "hdfs:\/\/"
    then
        # Make directory of destination file
        filedir=`dirname ${SPARK_SPARKWORDCOUNT_FILE}`
        command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -mkdir -p ${filedir}"
        echo "Running $command" >&2
        $command

        if echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^file:\/\/"
        then
            command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -copyFromLocal ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
        elif echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^hdfs:\/\/"
        then
            command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -cp ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
        fi
    elif echo ${SPARK_SPARKWORDCOUNT_FILE} | grep -q -E "alluxio:\/\/"
    then
        # Make directory of destination file
        filedir=`dirname ${SPARK_SPARKWORDCOUNT_FILE}`
        command="${ALLUXIO_HOME}/${alluxiocmdprefix}/alluxio fs mkdir ${filedir}"
        echo "Running $command" >&2
        $command

        if echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^file:\/\/"
        then
            # Get rid of file:// prefix, Alluxio doesn't support it
            filepath=$(echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | sed "s|file://||g")
            command="${ALLUXIO_HOME}/${alluxiocmdprefix}/alluxio fs copyFromLocal ${filepath} ${SPARK_SPARKWORDCOUNT_FILE}"
        elif echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^alluxio:\/\/"
        then
            command="${ALLUXIO_HOME}/${alluxiocmdprefix}/alluxio fs cp ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
        fi

    elif echo ${SPARK_SPARKWORDCOUNT_FILE} | grep -q -E "file:\/\/"
    then
        # Make directory of destination file
        filedir=`dirname ${SPARK_SPARKWORDCOUNT_FILE}`
        filedir=`echo ${filedir} | sed "s/^file:\/\///"`

        mkdir -p ${filedir}
        if [ $? -ne 0 ] ; then
            echo "mkdir failed making ${filedir}"
            exit 1
        fi

        if echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^file:\/\/"
        then
            sparkinputfile=`echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | sed "s/^file:\/\///"`
            sparkdestfile=`echo ${SPARK_SPARKWORDCOUNT_FILE} | sed "s/^file:\/\///"`
            command="cp ${sparkinputfile} ${sparkdestfile}"
        elif echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^hdfs:\/\/"
        then
            command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -copyToLocal ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
        fi
    fi

    echo "Running copy in $command" >&2
    $command
fi

if echo ${SPARK_VERSION} | grep -q -E "1\.[0-9]\.[0-9]"
then
    sparkexamplesjar=`find ${SPARK_HOME}/lib | grep jar | grep examples`

    command="${sparkcmdprefix}/spark-submit --class ${sparkwordcountclass} ${sparkexamplesjar} ${SPARK_SPARKWORDCOUNT_FILE}"
elif echo ${SPARK_VERSION} | grep -q -E "[2-3]\.[0-9]\.[0-9]"
then
    sparkexamplesjar=`find ${SPARK_HOME}/examples/jars/ | grep spark-examples`

    command="${sparkcmdprefix}/spark-submit --class ${sparkwordcountclass} ${sparkexamplesjar} ${SPARK_SPARKWORDCOUNT_FILE}"
fi

echo "Running $command" >&2
$command

exit 0
